
<!DOCTYPE HTML>
<html lang="zh-hans" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>示例: 并发的Web爬虫 · Go语言圣经</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-katex/katex.min.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    <link rel="next" href="ch8-07.html" />
    
    
    <link rel="prev" href="ch8-05.html" />
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    前言
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.1.1" data-path="../ch0/ch0-01.html">
            
                <a href="../ch0/ch0-01.html">
            
                    
                    Go语言起源
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.1.2" data-path="../ch0/ch0-02.html">
            
                <a href="../ch0/ch0-02.html">
            
                    
                    Go语言项目
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.1.3" data-path="../ch0/ch0-03.html">
            
                <a href="../ch0/ch0-03.html">
            
                    
                    本书的组织
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.1.4" data-path="../ch0/ch0-04.html">
            
                <a href="../ch0/ch0-04.html">
            
                    
                    更多的信息
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.1.5" data-path="../ch0/ch0-05.html">
            
                <a href="../ch0/ch0-05.html">
            
                    
                    致谢
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="../ch1/ch1.html">
            
                <a href="../ch1/ch1.html">
            
                    
                    入门
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.2.1" data-path="../ch1/ch1-01.html">
            
                <a href="../ch1/ch1-01.html">
            
                    
                    Hello, World
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.2" data-path="../ch1/ch1-02.html">
            
                <a href="../ch1/ch1-02.html">
            
                    
                    命令行参数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.3" data-path="../ch1/ch1-03.html">
            
                <a href="../ch1/ch1-03.html">
            
                    
                    查找重复的行
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.4" data-path="../ch1/ch1-04.html">
            
                <a href="../ch1/ch1-04.html">
            
                    
                    GIF动画
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.5" data-path="../ch1/ch1-05.html">
            
                <a href="../ch1/ch1-05.html">
            
                    
                    获取URL
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.6" data-path="../ch1/ch1-06.html">
            
                <a href="../ch1/ch1-06.html">
            
                    
                    并发获取多个URL
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.7" data-path="../ch1/ch1-07.html">
            
                <a href="../ch1/ch1-07.html">
            
                    
                    Web服务
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.8" data-path="../ch1/ch1-08.html">
            
                <a href="../ch1/ch1-08.html">
            
                    
                    本章要点
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="../ch2/ch2.html">
            
                <a href="../ch2/ch2.html">
            
                    
                    程序结构
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.3.1" data-path="../ch2/ch2-01.html">
            
                <a href="../ch2/ch2-01.html">
            
                    
                    命名
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.2" data-path="../ch2/ch2-02.html">
            
                <a href="../ch2/ch2-02.html">
            
                    
                    声明
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.3" data-path="../ch2/ch2-03.html">
            
                <a href="../ch2/ch2-03.html">
            
                    
                    变量
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.4" data-path="../ch2/ch2-04.html">
            
                <a href="../ch2/ch2-04.html">
            
                    
                    赋值
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.5" data-path="../ch2/ch2-05.html">
            
                <a href="../ch2/ch2-05.html">
            
                    
                    类型
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.6" data-path="../ch2/ch2-06.html">
            
                <a href="../ch2/ch2-06.html">
            
                    
                    包和文件
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.7" data-path="../ch2/ch2-07.html">
            
                <a href="../ch2/ch2-07.html">
            
                    
                    作用域
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="../ch3/ch3.html">
            
                <a href="../ch3/ch3.html">
            
                    
                    基础数据类型
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.4.1" data-path="../ch3/ch3-01.html">
            
                <a href="../ch3/ch3-01.html">
            
                    
                    整型
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.2" data-path="../ch3/ch3-02.html">
            
                <a href="../ch3/ch3-02.html">
            
                    
                    浮点数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.3" data-path="../ch3/ch3-03.html">
            
                <a href="../ch3/ch3-03.html">
            
                    
                    复数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.4" data-path="../ch3/ch3-04.html">
            
                <a href="../ch3/ch3-04.html">
            
                    
                    布尔型
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.5" data-path="../ch3/ch3-05.html">
            
                <a href="../ch3/ch3-05.html">
            
                    
                    字符串
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.6" data-path="../ch3/ch3-06.html">
            
                <a href="../ch3/ch3-06.html">
            
                    
                    常量
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.5" data-path="../ch4/ch4.html">
            
                <a href="../ch4/ch4.html">
            
                    
                    复合数据类型
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.5.1" data-path="../ch4/ch4-01.html">
            
                <a href="../ch4/ch4-01.html">
            
                    
                    数组
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.2" data-path="../ch4/ch4-02.html">
            
                <a href="../ch4/ch4-02.html">
            
                    
                    Slice
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.3" data-path="../ch4/ch4-03.html">
            
                <a href="../ch4/ch4-03.html">
            
                    
                    Map
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.4" data-path="../ch4/ch4-04.html">
            
                <a href="../ch4/ch4-04.html">
            
                    
                    结构体
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.5" data-path="../ch4/ch4-05.html">
            
                <a href="../ch4/ch4-05.html">
            
                    
                    JSON
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.6" data-path="../ch4/ch4-06.html">
            
                <a href="../ch4/ch4-06.html">
            
                    
                    文本和HTML模板
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.6" data-path="../ch5/ch5.html">
            
                <a href="../ch5/ch5.html">
            
                    
                    函数
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.6.1" data-path="../ch5/ch5-01.html">
            
                <a href="../ch5/ch5-01.html">
            
                    
                    函数声明
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.2" data-path="../ch5/ch5-02.html">
            
                <a href="../ch5/ch5-02.html">
            
                    
                    递归
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.3" data-path="../ch5/ch5-03.html">
            
                <a href="../ch5/ch5-03.html">
            
                    
                    多返回值
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.4" data-path="../ch5/ch5-04.html">
            
                <a href="../ch5/ch5-04.html">
            
                    
                    错误
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.5" data-path="../ch5/ch5-05.html">
            
                <a href="../ch5/ch5-05.html">
            
                    
                    函数值
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.6" data-path="../ch5/ch5-06.html">
            
                <a href="../ch5/ch5-06.html">
            
                    
                    匿名函数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.7" data-path="../ch5/ch5-07.html">
            
                <a href="../ch5/ch5-07.html">
            
                    
                    可变参数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.8" data-path="../ch5/ch5-08.html">
            
                <a href="../ch5/ch5-08.html">
            
                    
                    Deferred函数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.9" data-path="../ch5/ch5-09.html">
            
                <a href="../ch5/ch5-09.html">
            
                    
                    Panic异常
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.6.10" data-path="../ch5/ch5-10.html">
            
                <a href="../ch5/ch5-10.html">
            
                    
                    Recover捕获异常
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.7" data-path="../ch6/ch6.html">
            
                <a href="../ch6/ch6.html">
            
                    
                    方法
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.7.1" data-path="../ch6/ch6-01.html">
            
                <a href="../ch6/ch6-01.html">
            
                    
                    方法声明
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.7.2" data-path="../ch6/ch6-02.html">
            
                <a href="../ch6/ch6-02.html">
            
                    
                    基于指针对象的方法
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.7.3" data-path="../ch6/ch6-03.html">
            
                <a href="../ch6/ch6-03.html">
            
                    
                    通过嵌入结构体来扩展类型
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.7.4" data-path="../ch6/ch6-04.html">
            
                <a href="../ch6/ch6-04.html">
            
                    
                    方法值和方法表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.7.5" data-path="../ch6/ch6-05.html">
            
                <a href="../ch6/ch6-05.html">
            
                    
                    示例: Bit数组
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.7.6" data-path="../ch6/ch6-06.html">
            
                <a href="../ch6/ch6-06.html">
            
                    
                    封装
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.8" data-path="../ch7/ch7.html">
            
                <a href="../ch7/ch7.html">
            
                    
                    接口
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.8.1" data-path="../ch7/ch7-01.html">
            
                <a href="../ch7/ch7-01.html">
            
                    
                    接口是合约
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.2" data-path="../ch7/ch7-02.html">
            
                <a href="../ch7/ch7-02.html">
            
                    
                    接口类型
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.3" data-path="../ch7/ch7-03.html">
            
                <a href="../ch7/ch7-03.html">
            
                    
                    实现接口的条件
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.4" data-path="../ch7/ch7-04.html">
            
                <a href="../ch7/ch7-04.html">
            
                    
                    flag.Value接口
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.5" data-path="../ch7/ch7-05.html">
            
                <a href="../ch7/ch7-05.html">
            
                    
                    接口值
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.6" data-path="../ch7/ch7-06.html">
            
                <a href="../ch7/ch7-06.html">
            
                    
                    sort.Interface接口
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.7" data-path="../ch7/ch7-07.html">
            
                <a href="../ch7/ch7-07.html">
            
                    
                    http.Handler接口
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.8" data-path="../ch7/ch7-08.html">
            
                <a href="../ch7/ch7-08.html">
            
                    
                    error接口
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.9" data-path="../ch7/ch7-09.html">
            
                <a href="../ch7/ch7-09.html">
            
                    
                    示例: 表达式求值
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.10" data-path="../ch7/ch7-10.html">
            
                <a href="../ch7/ch7-10.html">
            
                    
                    类型断言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.11" data-path="../ch7/ch7-11.html">
            
                <a href="../ch7/ch7-11.html">
            
                    
                    基于类型断言识别错误类型
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.12" data-path="../ch7/ch7-12.html">
            
                <a href="../ch7/ch7-12.html">
            
                    
                    通过类型断言查询接口
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.13" data-path="../ch7/ch7-13.html">
            
                <a href="../ch7/ch7-13.html">
            
                    
                    类型分支
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.14" data-path="../ch7/ch7-14.html">
            
                <a href="../ch7/ch7-14.html">
            
                    
                    示例: 基于标记的XML解码
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.8.15" data-path="../ch7/ch7-15.html">
            
                <a href="../ch7/ch7-15.html">
            
                    
                    补充几点
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.9" data-path="ch8.html">
            
                <a href="ch8.html">
            
                    
                    Goroutines和Channels
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.9.1" data-path="ch8-01.html">
            
                <a href="ch8-01.html">
            
                    
                    Goroutines
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.2" data-path="ch8-02.html">
            
                <a href="ch8-02.html">
            
                    
                    示例: 并发的Clock服务
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.3" data-path="ch8-03.html">
            
                <a href="ch8-03.html">
            
                    
                    示例: 并发的Echo服务
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.4" data-path="ch8-04.html">
            
                <a href="ch8-04.html">
            
                    
                    Channels
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.5" data-path="ch8-05.html">
            
                <a href="ch8-05.html">
            
                    
                    并发的循环
            
                </a>
            

            
        </li>
    
        <li class="chapter active" data-level="1.9.6" data-path="ch8-06.html">
            
                <a href="ch8-06.html">
            
                    
                    示例: 并发的Web爬虫
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.7" data-path="ch8-07.html">
            
                <a href="ch8-07.html">
            
                    
                    基于select的多路复用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.8" data-path="ch8-08.html">
            
                <a href="ch8-08.html">
            
                    
                    示例: 并发的目录遍历
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.9" data-path="ch8-09.html">
            
                <a href="ch8-09.html">
            
                    
                    并发的退出
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.9.10" data-path="ch8-10.html">
            
                <a href="ch8-10.html">
            
                    
                    示例: 聊天服务
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.10" data-path="../ch9/ch9.html">
            
                <a href="../ch9/ch9.html">
            
                    
                    基于共享变量的并发
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.10.1" data-path="../ch9/ch9-01.html">
            
                <a href="../ch9/ch9-01.html">
            
                    
                    竞争条件
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.10.2" data-path="../ch9/ch9-02.html">
            
                <a href="../ch9/ch9-02.html">
            
                    
                    sync.Mutex互斥锁
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.10.3" data-path="../ch9/ch9-03.html">
            
                <a href="../ch9/ch9-03.html">
            
                    
                    sync.RWMutex读写锁
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.10.4" data-path="../ch9/ch9-04.html">
            
                <a href="../ch9/ch9-04.html">
            
                    
                    内存同步
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.10.5" data-path="../ch9/ch9-05.html">
            
                <a href="../ch9/ch9-05.html">
            
                    
                    sync.Once惰性初始化
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.10.6" data-path="../ch9/ch9-06.html">
            
                <a href="../ch9/ch9-06.html">
            
                    
                    竞争条件检测
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.10.7" data-path="../ch9/ch9-07.html">
            
                <a href="../ch9/ch9-07.html">
            
                    
                    示例: 并发的非阻塞缓存
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.10.8" data-path="../ch9/ch9-08.html">
            
                <a href="../ch9/ch9-08.html">
            
                    
                    Goroutines和线程
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.11" data-path="../ch10/ch10.html">
            
                <a href="../ch10/ch10.html">
            
                    
                    包和工具
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.11.1" data-path="../ch10/ch10-01.html">
            
                <a href="../ch10/ch10-01.html">
            
                    
                    包简介
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.11.2" data-path="../ch10/ch10-02.html">
            
                <a href="../ch10/ch10-02.html">
            
                    
                    导入路径
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.11.3" data-path="../ch10/ch10-03.html">
            
                <a href="../ch10/ch10-03.html">
            
                    
                    包声明
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.11.4" data-path="../ch10/ch10-04.html">
            
                <a href="../ch10/ch10-04.html">
            
                    
                    导入声明
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.11.5" data-path="../ch10/ch10-05.html">
            
                <a href="../ch10/ch10-05.html">
            
                    
                    包的匿名导入
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.11.6" data-path="../ch10/ch10-06.html">
            
                <a href="../ch10/ch10-06.html">
            
                    
                    包和命名
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.11.7" data-path="../ch10/ch10-07.html">
            
                <a href="../ch10/ch10-07.html">
            
                    
                    工具
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.12" data-path="../ch11/ch11.html">
            
                <a href="../ch11/ch11.html">
            
                    
                    测试
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.12.1" data-path="../ch11/ch11-01.html">
            
                <a href="../ch11/ch11-01.html">
            
                    
                    go test
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.12.2" data-path="../ch11/ch11-02.html">
            
                <a href="../ch11/ch11-02.html">
            
                    
                    测试函数
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.12.3" data-path="../ch11/ch11-03.html">
            
                <a href="../ch11/ch11-03.html">
            
                    
                    测试覆盖率
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.12.4" data-path="../ch11/ch11-04.html">
            
                <a href="../ch11/ch11-04.html">
            
                    
                    基准测试
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.12.5" data-path="../ch11/ch11-05.html">
            
                <a href="../ch11/ch11-05.html">
            
                    
                    剖析
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.12.6" data-path="../ch11/ch11-06.html">
            
                <a href="../ch11/ch11-06.html">
            
                    
                    示例函数
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.13" data-path="../ch12/ch12.html">
            
                <a href="../ch12/ch12.html">
            
                    
                    反射
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.13.1" data-path="../ch12/ch12-01.html">
            
                <a href="../ch12/ch12-01.html">
            
                    
                    为何需要反射?
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.2" data-path="../ch12/ch12-02.html">
            
                <a href="../ch12/ch12-02.html">
            
                    
                    reflect.Type和reflect.Value
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.3" data-path="../ch12/ch12-03.html">
            
                <a href="../ch12/ch12-03.html">
            
                    
                    Display递归打印
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.4" data-path="../ch12/ch12-04.html">
            
                <a href="../ch12/ch12-04.html">
            
                    
                    示例: 编码S表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.5" data-path="../ch12/ch12-05.html">
            
                <a href="../ch12/ch12-05.html">
            
                    
                    通过reflect.Value修改值
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.6" data-path="../ch12/ch12-06.html">
            
                <a href="../ch12/ch12-06.html">
            
                    
                    示例: 解码S表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.7" data-path="../ch12/ch12-07.html">
            
                <a href="../ch12/ch12-07.html">
            
                    
                    获取结构体字段标签
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.8" data-path="../ch12/ch12-08.html">
            
                <a href="../ch12/ch12-08.html">
            
                    
                    显示一个类型的方法集
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.13.9" data-path="../ch12/ch12-09.html">
            
                <a href="../ch12/ch12-09.html">
            
                    
                    几点忠告
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.14" data-path="../ch13/ch13.html">
            
                <a href="../ch13/ch13.html">
            
                    
                    底层编程
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.14.1" data-path="../ch13/ch13-01.html">
            
                <a href="../ch13/ch13-01.html">
            
                    
                    unsafe.Sizeof, Alignof 和 Offsetof
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.14.2" data-path="../ch13/ch13-02.html">
            
                <a href="../ch13/ch13-02.html">
            
                    
                    unsafe.Pointer
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.14.3" data-path="../ch13/ch13-03.html">
            
                <a href="../ch13/ch13-03.html">
            
                    
                    示例: 深度相等判断
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.14.4" data-path="../ch13/ch13-04.html">
            
                <a href="../ch13/ch13-04.html">
            
                    
                    通过cgo调用C代码
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.14.5" data-path="../ch13/ch13-05.html">
            
                <a href="../ch13/ch13-05.html">
            
                    
                    几点忠告
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.15" data-path="../appendix/appendix.html">
            
                <a href="../appendix/appendix.html">
            
                    
                    附录
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.15.1" data-path="../appendix/appendix-a-errata.html">
            
                <a href="../appendix/appendix-a-errata.html">
            
                    
                    附录A：原文勘误
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.15.2" data-path="../appendix/appendix-b-author.html">
            
                <a href="../appendix/appendix-b-author.html">
            
                    
                    附录B：作者译者
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.15.3" data-path="../appendix/appendix-c-cpoyright.html">
            
                <a href="../appendix/appendix-c-cpoyright.html">
            
                    
                    附录C：译文授权
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.15.4" data-path="../appendix/appendix-d-translations.html">
            
                <a href="../appendix/appendix-d-translations.html">
            
                    
                    附录D：其它语言
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            本书使用 GitBook 发布
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >示例: 并发的Web爬虫</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
                                <section class="normal markdown-section">
                                
                                <h2 id="86-&#x793A;&#x4F8B;-&#x5E76;&#x53D1;&#x7684;web&#x722C;&#x866B;">8.6. &#x793A;&#x4F8B;: &#x5E76;&#x53D1;&#x7684;Web&#x722C;&#x866B;</h2>
<p>&#x5728;5.6&#x8282;&#x4E2D;&#xFF0C;&#x6211;&#x4EEC;&#x505A;&#x4E86;&#x4E00;&#x4E2A;&#x7B80;&#x5355;&#x7684;web&#x722C;&#x866B;&#xFF0C;&#x7528;bfs(&#x5E7F;&#x5EA6;&#x4F18;&#x5148;)&#x7B97;&#x6CD5;&#x6765;&#x6293;&#x53D6;&#x6574;&#x4E2A;&#x7F51;&#x7AD9;&#x3002;&#x5728;&#x672C;&#x8282;&#x4E2D;&#xFF0C;&#x6211;&#x4EEC;&#x4F1A;&#x8BA9;&#x8FD9;&#x4E2A;&#x722C;&#x866B;&#x5E76;&#x884C;&#x5316;&#xFF0C;&#x8FD9;&#x6837;&#x6BCF;&#x4E00;&#x4E2A;&#x5F7C;&#x6B64;&#x72EC;&#x7ACB;&#x7684;&#x6293;&#x53D6;&#x547D;&#x4EE4;&#x53EF;&#x4EE5;&#x5E76;&#x884C;&#x8FDB;&#x884C;IO&#xFF0C;&#x6700;&#x5927;&#x5316;&#x5229;&#x7528;&#x7F51;&#x7EDC;&#x8D44;&#x6E90;&#x3002;crawl&#x51FD;&#x6570;&#x548C;gopl.io/ch5/findlinks3&#x4E2D;&#x7684;&#x662F;&#x4E00;&#x6837;&#x7684;&#x3002;</p>
<p><u><i>gopl.io/ch8/crawl1</i></u></p>
<pre><code class="lang-go"><span class="hljs-keyword">func</span> crawl(url <span class="hljs-keyword">string</span>) []<span class="hljs-keyword">string</span> {
    fmt.Println(url)
    list, err := links.Extract(url)
    <span class="hljs-keyword">if</span> err != <span class="hljs-literal">nil</span> {
        log.Print(err)
    }
    <span class="hljs-keyword">return</span> list
}
</code></pre>
<p>&#x4E3B;&#x51FD;&#x6570;&#x548C;5.6&#x8282;&#x4E2D;&#x7684;breadthFirst(&#x5E7F;&#x5EA6;&#x4F18;&#x5148;)&#x7C7B;&#x4F3C;&#x3002;&#x50CF;&#x4E4B;&#x524D;&#x4E00;&#x6837;&#xFF0C;&#x4E00;&#x4E2A;worklist&#x662F;&#x4E00;&#x4E2A;&#x8BB0;&#x5F55;&#x4E86;&#x9700;&#x8981;&#x5904;&#x7406;&#x7684;&#x5143;&#x7D20;&#x7684;&#x961F;&#x5217;&#xFF0C;&#x6BCF;&#x4E00;&#x4E2A;&#x5143;&#x7D20;&#x90FD;&#x662F;&#x4E00;&#x4E2A;&#x9700;&#x8981;&#x6293;&#x53D6;&#x7684;URL&#x5217;&#x8868;&#xFF0C;&#x4E0D;&#x8FC7;&#x8FD9;&#x4E00;&#x6B21;&#x6211;&#x4EEC;&#x7528;channel&#x4EE3;&#x66FF;slice&#x6765;&#x505A;&#x8FD9;&#x4E2A;&#x961F;&#x5217;&#x3002;&#x6BCF;&#x4E00;&#x4E2A;&#x5BF9;crawl&#x7684;&#x8C03;&#x7528;&#x90FD;&#x4F1A;&#x5728;&#x4ED6;&#x4EEC;&#x81EA;&#x5DF1;&#x7684;goroutine&#x4E2D;&#x8FDB;&#x884C;&#x5E76;&#x4E14;&#x4F1A;&#x628A;&#x4ED6;&#x4EEC;&#x6293;&#x5230;&#x7684;&#x94FE;&#x63A5;&#x53D1;&#x9001;&#x56DE;worklist&#x3002;</p>
<pre><code class="lang-go"><span class="hljs-keyword">func</span> main() {
    worklist := <span class="hljs-built_in">make</span>(<span class="hljs-keyword">chan</span> []<span class="hljs-keyword">string</span>)

    <span class="hljs-comment">// Start with the command-line arguments.</span>
    <span class="hljs-keyword">go</span> <span class="hljs-keyword">func</span>() { worklist &lt;- os.Args[<span class="hljs-number">1</span>:] }()

    <span class="hljs-comment">// Crawl the web concurrently.</span>
    seen := <span class="hljs-built_in">make</span>(<span class="hljs-keyword">map</span>[<span class="hljs-keyword">string</span>]<span class="hljs-keyword">bool</span>)
    <span class="hljs-keyword">for</span> list := <span class="hljs-keyword">range</span> worklist {
        <span class="hljs-keyword">for</span> _, link := <span class="hljs-keyword">range</span> list {
            <span class="hljs-keyword">if</span> !seen[link] {
                seen[link] = <span class="hljs-literal">true</span>
                <span class="hljs-keyword">go</span> <span class="hljs-keyword">func</span>(link <span class="hljs-keyword">string</span>) {
                    worklist &lt;- crawl(link)
                }(link)
            }
        }
    }
}
</code></pre>
<p>&#x6CE8;&#x610F;&#x8FD9;&#x91CC;&#x7684;crawl&#x6240;&#x5728;&#x7684;goroutine&#x4F1A;&#x5C06;link&#x4F5C;&#x4E3A;&#x4E00;&#x4E2A;&#x663E;&#x5F0F;&#x7684;&#x53C2;&#x6570;&#x4F20;&#x5165;&#xFF0C;&#x6765;&#x907F;&#x514D;&#x201C;&#x5FAA;&#x73AF;&#x53D8;&#x91CF;&#x5FEB;&#x7167;&#x201D;&#x7684;&#x95EE;&#x9898;&#xFF08;&#x5728;5.6.1&#x4E2D;&#x6709;&#x8BB2;&#x89E3;&#xFF09;&#x3002;&#x53E6;&#x5916;&#x6CE8;&#x610F;&#x8FD9;&#x91CC;&#x5C06;&#x547D;&#x4EE4;&#x884C;&#x53C2;&#x6570;&#x4F20;&#x5165;worklist&#x4E5F;&#x662F;&#x5728;&#x4E00;&#x4E2A;&#x53E6;&#x5916;&#x7684;goroutine&#x4E2D;&#x8FDB;&#x884C;&#x7684;&#xFF0C;&#x8FD9;&#x662F;&#x4E3A;&#x4E86;&#x907F;&#x514D;channel&#x4E24;&#x7AEF;&#x7684;main goroutine&#x4E0E;crawler goroutine&#x90FD;&#x5C1D;&#x8BD5;&#x5411;&#x5BF9;&#x65B9;&#x53D1;&#x9001;&#x5185;&#x5BB9;&#xFF0C;&#x5374;&#x6CA1;&#x6709;&#x4E00;&#x7AEF;&#x63A5;&#x6536;&#x5185;&#x5BB9;&#x65F6;&#x53D1;&#x751F;&#x6B7B;&#x9501;&#x3002;&#x5F53;&#x7136;&#xFF0C;&#x8FD9;&#x91CC;&#x6211;&#x4EEC;&#x4E5F;&#x53EF;&#x4EE5;&#x7528;buffered channel&#x6765;&#x89E3;&#x51B3;&#x95EE;&#x9898;&#xFF0C;&#x8FD9;&#x91CC;&#x4E0D;&#x518D;&#x8D58;&#x8FF0;&#x3002;</p>
<p>&#x73B0;&#x5728;&#x722C;&#x866B;&#x53EF;&#x4EE5;&#x9AD8;&#x5E76;&#x53D1;&#x5730;&#x8FD0;&#x884C;&#x8D77;&#x6765;&#xFF0C;&#x5E76;&#x4E14;&#x53EF;&#x4EE5;&#x4EA7;&#x751F;&#x4E00;&#x5927;&#x5768;&#x7684;URL&#x4E86;&#xFF0C;&#x4E0D;&#x8FC7;&#x8FD8;&#x662F;&#x4F1A;&#x6709;&#x4FE9;&#x95EE;&#x9898;&#x3002;&#x4E00;&#x4E2A;&#x95EE;&#x9898;&#x662F;&#x5728;&#x8FD0;&#x884C;&#x4E00;&#x6BB5;&#x65F6;&#x95F4;&#x540E;&#x53EF;&#x80FD;&#x4F1A;&#x51FA;&#x73B0;&#x5728;log&#x7684;&#x9519;&#x8BEF;&#x4FE1;&#x606F;&#x91CC;&#x7684;&#xFF1A;</p>
<pre><code>$ go build gopl.io/ch8/crawl1
$ ./crawl1 http://gopl.io/
http://gopl.io/
https://golang.org/help/
https://golang.org/doc/
https://golang.org/blog/
...
2015/07/15 18:22:12 Get ...: dial tcp: lookup blog.golang.org: no such host
2015/07/15 18:22:12 Get ...: dial tcp 23.21.222.120:443: socket: too many open files
...
</code></pre><p>&#x6700;&#x521D;&#x7684;&#x9519;&#x8BEF;&#x4FE1;&#x606F;&#x662F;&#x4E00;&#x4E2A;&#x8BA9;&#x4EBA;&#x83AB;&#x540D;&#x7684;DNS&#x67E5;&#x627E;&#x5931;&#x8D25;&#xFF0C;&#x5373;&#x4F7F;&#x8FD9;&#x4E2A;&#x57DF;&#x540D;&#x662F;&#x5B8C;&#x5168;&#x53EF;&#x9760;&#x7684;&#x3002;&#x800C;&#x968F;&#x540E;&#x7684;&#x9519;&#x8BEF;&#x4FE1;&#x606F;&#x63ED;&#x793A;&#x4E86;&#x539F;&#x56E0;&#xFF1A;&#x8FD9;&#x4E2A;&#x7A0B;&#x5E8F;&#x4E00;&#x6B21;&#x6027;&#x521B;&#x5EFA;&#x4E86;&#x592A;&#x591A;&#x7F51;&#x7EDC;&#x8FDE;&#x63A5;&#xFF0C;&#x8D85;&#x8FC7;&#x4E86;&#x6BCF;&#x4E00;&#x4E2A;&#x8FDB;&#x7A0B;&#x7684;&#x6253;&#x5F00;&#x6587;&#x4EF6;&#x6570;&#x9650;&#x5236;&#xFF0C;&#x65E2;&#x800C;&#x5BFC;&#x81F4;&#x4E86;&#x5728;&#x8C03;&#x7528;net.Dial&#x50CF;DNS&#x67E5;&#x627E;&#x5931;&#x8D25;&#x8FD9;&#x6837;&#x7684;&#x95EE;&#x9898;&#x3002;</p>
<p>&#x8FD9;&#x4E2A;&#x7A0B;&#x5E8F;&#x5B9E;&#x5728;&#x662F;&#x592A;&#x4ED6;&#x5988;&#x5E76;&#x884C;&#x4E86;&#x3002;&#x65E0;&#x7A77;&#x65E0;&#x5C3D;&#x5730;&#x5E76;&#x884C;&#x5316;&#x5E76;&#x4E0D;&#x662F;&#x4EC0;&#x4E48;&#x597D;&#x4E8B;&#x60C5;&#xFF0C;&#x56E0;&#x4E3A;&#x4E0D;&#x7BA1;&#x600E;&#x4E48;&#x8BF4;&#xFF0C;&#x4F60;&#x7684;&#x7CFB;&#x7EDF;&#x603B;&#x662F;&#x4F1A;&#x6709;&#x4E00;&#x4E9B;&#x4E2A;&#x9650;&#x5236;&#x56E0;&#x7D20;&#xFF0C;&#x6BD4;&#x5982;CPU&#x6838;&#x5FC3;&#x6570;&#x4F1A;&#x9650;&#x5236;&#x4F60;&#x7684;&#x8BA1;&#x7B97;&#x8D1F;&#x8F7D;&#xFF0C;&#x6BD4;&#x5982;&#x4F60;&#x7684;&#x786C;&#x76D8;&#x8F6C;&#x8F74;&#x548C;&#x78C1;&#x5934;&#x6570;&#x9650;&#x5236;&#x4E86;&#x4F60;&#x7684;&#x672C;&#x5730;&#x78C1;&#x76D8;IO&#x64CD;&#x4F5C;&#x9891;&#x7387;&#xFF0C;&#x6BD4;&#x5982;&#x4F60;&#x7684;&#x7F51;&#x7EDC;&#x5E26;&#x5BBD;&#x9650;&#x5236;&#x4E86;&#x4F60;&#x7684;&#x4E0B;&#x8F7D;&#x901F;&#x5EA6;&#x4E0A;&#x9650;&#xFF0C;&#x6216;&#x8005;&#x662F;&#x4F60;&#x7684;&#x4E00;&#x4E2A;web&#x670D;&#x52A1;&#x7684;&#x670D;&#x52A1;&#x5BB9;&#x91CF;&#x4E0A;&#x9650;&#x7B49;&#x7B49;&#x3002;&#x4E3A;&#x4E86;&#x89E3;&#x51B3;&#x8FD9;&#x4E2A;&#x95EE;&#x9898;&#xFF0C;&#x6211;&#x4EEC;&#x53EF;&#x4EE5;&#x9650;&#x5236;&#x5E76;&#x53D1;&#x7A0B;&#x5E8F;&#x6240;&#x4F7F;&#x7528;&#x7684;&#x8D44;&#x6E90;&#x6765;&#x4F7F;&#x4E4B;&#x9002;&#x5E94;&#x81EA;&#x5DF1;&#x7684;&#x8FD0;&#x884C;&#x73AF;&#x5883;&#x3002;&#x5BF9;&#x4E8E;&#x6211;&#x4EEC;&#x7684;&#x4F8B;&#x5B50;&#x6765;&#x8BF4;&#xFF0C;&#x6700;&#x7B80;&#x5355;&#x7684;&#x65B9;&#x6CD5;&#x5C31;&#x662F;&#x9650;&#x5236;&#x5BF9;links.Extract&#x5728;&#x540C;&#x4E00;&#x65F6;&#x95F4;&#x6700;&#x591A;&#x4E0D;&#x4F1A;&#x6709;&#x8D85;&#x8FC7;n&#x6B21;&#x8C03;&#x7528;&#xFF0C;&#x8FD9;&#x91CC;&#x7684;n&#x4E00;&#x822C;&#x5C0F;&#x4E8E;&#x6587;&#x4EF6;&#x63CF;&#x8FF0;&#x7B26;&#x7684;&#x4E0A;&#x9650;&#x503C;&#xFF0C;&#x6BD4;&#x5982;20&#x3002;&#x8FD9;&#x548C;&#x4E00;&#x4E2A;&#x591C;&#x5E97;&#x91CC;&#x9650;&#x5236;&#x5BA2;&#x4EBA;&#x6570;&#x76EE;&#x662F;&#x4E00;&#x4E2A;&#x9053;&#x7406;&#xFF0C;&#x53EA;&#x6709;&#x5F53;&#x6709;&#x5BA2;&#x4EBA;&#x79BB;&#x5F00;&#x65F6;&#xFF0C;&#x624D;&#x4F1A;&#x5141;&#x8BB8;&#x65B0;&#x7684;&#x5BA2;&#x4EBA;&#x8FDB;&#x5165;&#x5E97;&#x5185;&#x3002;</p>
<p>&#x6211;&#x4EEC;&#x53EF;&#x4EE5;&#x7528;&#x4E00;&#x4E2A;&#x6709;&#x5BB9;&#x91CF;&#x9650;&#x5236;&#x7684;buffered channel&#x6765;&#x63A7;&#x5236;&#x5E76;&#x53D1;&#xFF0C;&#x8FD9;&#x7C7B;&#x4F3C;&#x4E8E;&#x64CD;&#x4F5C;&#x7CFB;&#x7EDF;&#x91CC;&#x7684;&#x8BA1;&#x6570;&#x4FE1;&#x53F7;&#x91CF;&#x6982;&#x5FF5;&#x3002;&#x4ECE;&#x6982;&#x5FF5;&#x4E0A;&#x8BB2;&#xFF0C;channel&#x91CC;&#x7684;n&#x4E2A;&#x7A7A;&#x69FD;&#x4EE3;&#x8868;n&#x4E2A;&#x53EF;&#x4EE5;&#x5904;&#x7406;&#x5185;&#x5BB9;&#x7684;token&#xFF08;&#x901A;&#x884C;&#x8BC1;&#xFF09;&#xFF0C;&#x4ECE;channel&#x91CC;&#x63A5;&#x6536;&#x4E00;&#x4E2A;&#x503C;&#x4F1A;&#x91CA;&#x653E;&#x5176;&#x4E2D;&#x7684;&#x4E00;&#x4E2A;token&#xFF0C;&#x5E76;&#x4E14;&#x751F;&#x6210;&#x4E00;&#x4E2A;&#x65B0;&#x7684;&#x7A7A;&#x69FD;&#x4F4D;&#x3002;&#x8FD9;&#x6837;&#x4FDD;&#x8BC1;&#x4E86;&#x5728;&#x6CA1;&#x6709;&#x63A5;&#x6536;&#x4ECB;&#x5165;&#x65F6;&#x6700;&#x591A;&#x6709;n&#x4E2A;&#x53D1;&#x9001;&#x64CD;&#x4F5C;&#x3002;&#xFF08;&#x8FD9;&#x91CC;&#x53EF;&#x80FD;&#x6211;&#x4EEC;&#x62FF;channel&#x91CC;&#x586B;&#x5145;&#x7684;&#x69FD;&#x6765;&#x505A;token&#x66F4;&#x76F4;&#x89C2;&#x4E00;&#x4E9B;&#xFF0C;&#x4E0D;&#x8FC7;&#x8FD8;&#x662F;&#x8FD9;&#x6837;&#x5427;&#x3002;&#xFF09;&#x7531;&#x4E8E;channel&#x91CC;&#x7684;&#x5143;&#x7D20;&#x7C7B;&#x578B;&#x5E76;&#x4E0D;&#x91CD;&#x8981;&#xFF0C;&#x6211;&#x4EEC;&#x7528;&#x4E00;&#x4E2A;&#x96F6;&#x503C;&#x7684;struct{}&#x6765;&#x4F5C;&#x4E3A;&#x5176;&#x5143;&#x7D20;&#x3002;</p>
<p>&#x8BA9;&#x6211;&#x4EEC;&#x91CD;&#x5199;crawl&#x51FD;&#x6570;&#xFF0C;&#x5C06;&#x5BF9;links.Extract&#x7684;&#x8C03;&#x7528;&#x64CD;&#x4F5C;&#x7528;&#x83B7;&#x53D6;&#x3001;&#x91CA;&#x653E;token&#x7684;&#x64CD;&#x4F5C;&#x5305;&#x88F9;&#x8D77;&#x6765;&#xFF0C;&#x6765;&#x786E;&#x4FDD;&#x540C;&#x4E00;&#x65F6;&#x95F4;&#x5BF9;&#x5176;&#x53EA;&#x6709;20&#x4E2A;&#x8C03;&#x7528;&#x3002;&#x4FE1;&#x53F7;&#x91CF;&#x6570;&#x91CF;&#x548C;&#x5176;&#x80FD;&#x64CD;&#x4F5C;&#x7684;IO&#x8D44;&#x6E90;&#x6570;&#x91CF;&#x5E94;&#x4FDD;&#x6301;&#x63A5;&#x8FD1;&#x3002;</p>
<p><u><i>gopl.io/ch8/crawl2</i></u></p>
<pre><code class="lang-go"><span class="hljs-comment">// tokens is a counting semaphore used to</span>
<span class="hljs-comment">// enforce a limit of 20 concurrent requests.</span>
<span class="hljs-keyword">var</span> tokens = <span class="hljs-built_in">make</span>(<span class="hljs-keyword">chan</span> <span class="hljs-keyword">struct</span>{}, <span class="hljs-number">20</span>)

<span class="hljs-keyword">func</span> crawl(url <span class="hljs-keyword">string</span>) []<span class="hljs-keyword">string</span> {
    fmt.Println(url)
    tokens &lt;- <span class="hljs-keyword">struct</span>{}{} <span class="hljs-comment">// acquire a token</span>
    list, err := links.Extract(url)
    &lt;-tokens <span class="hljs-comment">// release the token</span>
    <span class="hljs-keyword">if</span> err != <span class="hljs-literal">nil</span> {
        log.Print(err)
    }
    <span class="hljs-keyword">return</span> list
}
</code></pre>
<p>&#x7B2C;&#x4E8C;&#x4E2A;&#x95EE;&#x9898;&#x662F;&#x8FD9;&#x4E2A;&#x7A0B;&#x5E8F;&#x6C38;&#x8FDC;&#x90FD;&#x4E0D;&#x4F1A;&#x7EC8;&#x6B62;&#xFF0C;&#x5373;&#x4F7F;&#x5B83;&#x5DF2;&#x7ECF;&#x722C;&#x5230;&#x4E86;&#x6240;&#x6709;&#x521D;&#x59CB;&#x94FE;&#x63A5;&#x884D;&#x751F;&#x51FA;&#x7684;&#x94FE;&#x63A5;&#x3002;&#xFF08;&#x5F53;&#x7136;&#xFF0C;&#x9664;&#x975E;&#x4F60;&#x614E;&#x91CD;&#x5730;&#x9009;&#x62E9;&#x4E86;&#x5408;&#x9002;&#x7684;&#x521D;&#x59CB;&#x5316;URL&#x6216;&#x8005;&#x5DF2;&#x7ECF;&#x5B9E;&#x73B0;&#x4E86;&#x7EC3;&#x4E60;8.6&#x4E2D;&#x7684;&#x6DF1;&#x5EA6;&#x9650;&#x5236;&#xFF0C;&#x4F60;&#x5E94;&#x8BE5;&#x8FD8;&#x6CA1;&#x6709;&#x610F;&#x8BC6;&#x5230;&#x8FD9;&#x4E2A;&#x95EE;&#x9898;&#x3002;&#xFF09;&#x4E3A;&#x4E86;&#x4F7F;&#x8FD9;&#x4E2A;&#x7A0B;&#x5E8F;&#x80FD;&#x591F;&#x7EC8;&#x6B62;&#xFF0C;&#x6211;&#x4EEC;&#x9700;&#x8981;&#x5728;worklist&#x4E3A;&#x7A7A;&#x6216;&#x8005;&#x6CA1;&#x6709;crawl&#x7684;goroutine&#x5728;&#x8FD0;&#x884C;&#x65F6;&#x9000;&#x51FA;&#x4E3B;&#x5FAA;&#x73AF;&#x3002;</p>
<pre><code class="lang-go"><span class="hljs-keyword">func</span> main() {
    worklist := <span class="hljs-built_in">make</span>(<span class="hljs-keyword">chan</span> []<span class="hljs-keyword">string</span>)
    <span class="hljs-keyword">var</span> n <span class="hljs-keyword">int</span> <span class="hljs-comment">// number of pending sends to worklist</span>

    <span class="hljs-comment">// Start with the command-line arguments.</span>
    n++
    <span class="hljs-keyword">go</span> <span class="hljs-keyword">func</span>() { worklist &lt;- os.Args[<span class="hljs-number">1</span>:] }()

    <span class="hljs-comment">// Crawl the web concurrently.</span>
    seen := <span class="hljs-built_in">make</span>(<span class="hljs-keyword">map</span>[<span class="hljs-keyword">string</span>]<span class="hljs-keyword">bool</span>)

    <span class="hljs-keyword">for</span> ; n &gt; <span class="hljs-number">0</span>; n-- {
        list := &lt;-worklist
        <span class="hljs-keyword">for</span> _, link := <span class="hljs-keyword">range</span> list {
            <span class="hljs-keyword">if</span> !seen[link] {
                seen[link] = <span class="hljs-literal">true</span>
                n++
                <span class="hljs-keyword">go</span> <span class="hljs-keyword">func</span>(link <span class="hljs-keyword">string</span>) {
                    worklist &lt;- crawl(link)
                }(link)
            }
        }
    }
}
</code></pre>
<p>&#x8FD9;&#x4E2A;&#x7248;&#x672C;&#x4E2D;&#xFF0C;&#x8BA1;&#x6570;&#x5668;n&#x5BF9;worklist&#x7684;&#x53D1;&#x9001;&#x64CD;&#x4F5C;&#x6570;&#x91CF;&#x8FDB;&#x884C;&#x4E86;&#x9650;&#x5236;&#x3002;&#x6BCF;&#x4E00;&#x6B21;&#x6211;&#x4EEC;&#x53D1;&#x73B0;&#x6709;&#x5143;&#x7D20;&#x9700;&#x8981;&#x88AB;&#x53D1;&#x9001;&#x5230;worklist&#x65F6;&#xFF0C;&#x6211;&#x4EEC;&#x90FD;&#x4F1A;&#x5BF9;n&#x8FDB;&#x884C;++&#x64CD;&#x4F5C;&#xFF0C;&#x5728;&#x5411;worklist&#x4E2D;&#x53D1;&#x9001;&#x521D;&#x59CB;&#x7684;&#x547D;&#x4EE4;&#x884C;&#x53C2;&#x6570;&#x4E4B;&#x524D;&#xFF0C;&#x6211;&#x4EEC;&#x4E5F;&#x8FDB;&#x884C;&#x8FC7;&#x4E00;&#x6B21;++&#x64CD;&#x4F5C;&#x3002;&#x8FD9;&#x91CC;&#x7684;&#x64CD;&#x4F5C;++&#x662F;&#x5728;&#x6BCF;&#x542F;&#x52A8;&#x4E00;&#x4E2A;crawler&#x7684;goroutine&#x4E4B;&#x524D;&#x3002;&#x4E3B;&#x5FAA;&#x73AF;&#x4F1A;&#x5728;n&#x51CF;&#x4E3A;0&#x65F6;&#x7EC8;&#x6B62;&#xFF0C;&#x8FD9;&#x65F6;&#x5019;&#x8BF4;&#x660E;&#x6CA1;&#x6D3B;&#x53EF;&#x5E72;&#x4E86;&#x3002;</p>
<p>&#x73B0;&#x5728;&#x8FD9;&#x4E2A;&#x5E76;&#x53D1;&#x722C;&#x866B;&#x4F1A;&#x6BD4;5.6&#x8282;&#x4E2D;&#x7684;&#x6DF1;&#x5EA6;&#x4F18;&#x5148;&#x641C;&#x7D22;&#x7248;&#x5FEB;&#x4E0A;20&#x500D;&#xFF0C;&#x800C;&#x4E14;&#x4E0D;&#x4F1A;&#x51FA;&#x4EC0;&#x4E48;&#x9519;&#xFF0C;&#x5E76;&#x4E14;&#x5728;&#x5176;&#x5B8C;&#x6210;&#x4EFB;&#x52A1;&#x65F6;&#x4E5F;&#x4F1A;&#x6B63;&#x786E;&#x5730;&#x7EC8;&#x6B62;&#x3002;</p>
<p>&#x4E0B;&#x9762;&#x7684;&#x7A0B;&#x5E8F;&#x662F;&#x907F;&#x514D;&#x8FC7;&#x5EA6;&#x5E76;&#x53D1;&#x7684;&#x53E6;&#x4E00;&#x79CD;&#x601D;&#x8DEF;&#x3002;&#x8FD9;&#x4E2A;&#x7248;&#x672C;&#x4F7F;&#x7528;&#x4E86;&#x539F;&#x6765;&#x7684;crawl&#x51FD;&#x6570;&#xFF0C;&#x4F46;&#x6CA1;&#x6709;&#x4F7F;&#x7528;&#x8BA1;&#x6570;&#x4FE1;&#x53F7;&#x91CF;&#xFF0C;&#x53D6;&#x800C;&#x4EE3;&#x4E4B;&#x7528;&#x4E86;20&#x4E2A;&#x5E38;&#x9A7B;&#x7684;crawler goroutine&#xFF0C;&#x8FD9;&#x6837;&#x6765;&#x4FDD;&#x8BC1;&#x6700;&#x591A;20&#x4E2A;HTTP&#x8BF7;&#x6C42;&#x5728;&#x5E76;&#x53D1;&#x3002;</p>
<pre><code class="lang-go"><span class="hljs-keyword">func</span> main() {
    worklist := <span class="hljs-built_in">make</span>(<span class="hljs-keyword">chan</span> []<span class="hljs-keyword">string</span>)  <span class="hljs-comment">// lists of URLs, may have duplicates</span>
    unseenLinks := <span class="hljs-built_in">make</span>(<span class="hljs-keyword">chan</span> <span class="hljs-keyword">string</span>) <span class="hljs-comment">// de-duplicated URLs</span>

    <span class="hljs-comment">// Add command-line arguments to worklist.</span>
    <span class="hljs-keyword">go</span> <span class="hljs-keyword">func</span>() { worklist &lt;- os.Args[<span class="hljs-number">1</span>:] }()

    <span class="hljs-comment">// Create 20 crawler goroutines to fetch each unseen link.</span>
    <span class="hljs-keyword">for</span> i := <span class="hljs-number">0</span>; i &lt; <span class="hljs-number">20</span>; i++ {
        <span class="hljs-keyword">go</span> <span class="hljs-keyword">func</span>() {
            <span class="hljs-keyword">for</span> link := <span class="hljs-keyword">range</span> unseenLinks {
                foundLinks := crawl(link)
                <span class="hljs-keyword">go</span> <span class="hljs-keyword">func</span>() { worklist &lt;- foundLinks }()
            }
        }()
    }

    <span class="hljs-comment">// The main goroutine de-duplicates worklist items</span>
    <span class="hljs-comment">// and sends the unseen ones to the crawlers.</span>
    seen := <span class="hljs-built_in">make</span>(<span class="hljs-keyword">map</span>[<span class="hljs-keyword">string</span>]<span class="hljs-keyword">bool</span>)
    <span class="hljs-keyword">for</span> list := <span class="hljs-keyword">range</span> worklist {
        <span class="hljs-keyword">for</span> _, link := <span class="hljs-keyword">range</span> list {
            <span class="hljs-keyword">if</span> !seen[link] {
                seen[link] = <span class="hljs-literal">true</span>
                unseenLinks &lt;- link
            }
        }
    }
}
</code></pre>
<p>&#x6240;&#x6709;&#x7684;&#x722C;&#x866B;goroutine&#x73B0;&#x5728;&#x90FD;&#x662F;&#x88AB;&#x540C;&#x4E00;&#x4E2A;channel - unseenLinks&#x5582;&#x9971;&#x7684;&#x4E86;&#x3002;&#x4E3B;goroutine&#x8D1F;&#x8D23;&#x62C6;&#x5206;&#x5B83;&#x4ECE;worklist&#x91CC;&#x62FF;&#x5230;&#x7684;&#x5143;&#x7D20;&#xFF0C;&#x7136;&#x540E;&#x628A;&#x6CA1;&#x6709;&#x6293;&#x8FC7;&#x7684;&#x7ECF;&#x7531;unseenLinks channel&#x53D1;&#x9001;&#x7ED9;&#x4E00;&#x4E2A;&#x722C;&#x866B;&#x7684;goroutine&#x3002;</p>
<p>seen&#x8FD9;&#x4E2A;map&#x88AB;&#x9650;&#x5B9A;&#x5728;main goroutine&#x4E2D;&#xFF1B;&#x4E5F;&#x5C31;&#x662F;&#x8BF4;&#x8FD9;&#x4E2A;map&#x53EA;&#x80FD;&#x5728;main goroutine&#x4E2D;&#x8FDB;&#x884C;&#x8BBF;&#x95EE;&#x3002;&#x7C7B;&#x4F3C;&#x4E8E;&#x5176;&#x5B83;&#x7684;&#x4FE1;&#x606F;&#x9690;&#x85CF;&#x65B9;&#x5F0F;&#xFF0C;&#x8FD9;&#x6837;&#x7684;&#x7EA6;&#x675F;&#x53EF;&#x4EE5;&#x8BA9;&#x6211;&#x4EEC;&#x4ECE;&#x4E00;&#x5B9A;&#x7A0B;&#x5EA6;&#x4E0A;&#x4FDD;&#x8BC1;&#x7A0B;&#x5E8F;&#x7684;&#x6B63;&#x786E;&#x6027;&#x3002;&#x4F8B;&#x5982;&#xFF0C;&#x5185;&#x90E8;&#x53D8;&#x91CF;&#x4E0D;&#x80FD;&#x591F;&#x5728;&#x51FD;&#x6570;&#x5916;&#x90E8;&#x88AB;&#x8BBF;&#x95EE;&#x5230;&#xFF1B;&#x53D8;&#x91CF;&#xFF08;&#xA7;2.3.4&#xFF09;&#x5728;&#x6CA1;&#x6709;&#x53D1;&#x751F;&#x53D8;&#x91CF;&#x9003;&#x9038;&#xFF08;&#x8BD1;&#x6CE8;&#xFF1A;&#x5C40;&#x90E8;&#x53D8;&#x91CF;&#x88AB;&#x5168;&#x5C40;&#x53D8;&#x91CF;&#x5F15;&#x7528;&#x5730;&#x5740;&#x5BFC;&#x81F4;&#x53D8;&#x91CF;&#x88AB;&#x5206;&#x914D;&#x5728;&#x5806;&#x4E0A;&#xFF09;&#x7684;&#x60C5;&#x51B5;&#x4E0B;&#x662F;&#x65E0;&#x6CD5;&#x5728;&#x51FD;&#x6570;&#x5916;&#x90E8;&#x8BBF;&#x95EE;&#x7684;&#xFF1B;&#x4E00;&#x4E2A;&#x5BF9;&#x8C61;&#x7684;&#x5C01;&#x88C5;&#x5B57;&#x6BB5;&#x65E0;&#x6CD5;&#x88AB;&#x8BE5;&#x5BF9;&#x8C61;&#x7684;&#x65B9;&#x6CD5;&#x4EE5;&#x5916;&#x7684;&#x65B9;&#x6CD5;&#x8BBF;&#x95EE;&#x5230;&#x3002;&#x5728;&#x6240;&#x6709;&#x7684;&#x60C5;&#x51B5;&#x4E0B;&#xFF0C;&#x4FE1;&#x606F;&#x9690;&#x85CF;&#x90FD;&#x53EF;&#x4EE5;&#x5E2E;&#x52A9;&#x6211;&#x4EEC;&#x7EA6;&#x675F;&#x6211;&#x4EEC;&#x7684;&#x7A0B;&#x5E8F;&#xFF0C;&#x4F7F;&#x5176;&#x4E0D;&#x53D1;&#x751F;&#x610F;&#x6599;&#x4E4B;&#x5916;&#x7684;&#x60C5;&#x51B5;&#x3002;</p>
<p>crawl&#x51FD;&#x6570;&#x722C;&#x5230;&#x7684;&#x94FE;&#x63A5;&#x5728;&#x4E00;&#x4E2A;&#x4E13;&#x6709;&#x7684;goroutine&#x4E2D;&#x88AB;&#x53D1;&#x9001;&#x5230;worklist&#x4E2D;&#x6765;&#x907F;&#x514D;&#x6B7B;&#x9501;&#x3002;&#x4E3A;&#x4E86;&#x8282;&#x7701;&#x7BC7;&#x5E45;&#xFF0C;&#x8FD9;&#x4E2A;&#x4F8B;&#x5B50;&#x7684;&#x7EC8;&#x6B62;&#x95EE;&#x9898;&#x6211;&#x4EEC;&#x5148;&#x4E0D;&#x8FDB;&#x884C;&#x8BE6;&#x7EC6;&#x9610;&#x8FF0;&#x4E86;&#x3002;</p>
<p><strong>&#x7EC3;&#x4E60; 8.6&#xFF1A;</strong> &#x4E3A;&#x5E76;&#x53D1;&#x722C;&#x866B;&#x589E;&#x52A0;&#x6DF1;&#x5EA6;&#x9650;&#x5236;&#x3002;&#x4E5F;&#x5C31;&#x662F;&#x8BF4;&#xFF0C;&#x5982;&#x679C;&#x7528;&#x6237;&#x8BBE;&#x7F6E;&#x4E86;depth=3&#xFF0C;&#x90A3;&#x4E48;&#x53EA;&#x6709;&#x4ECE;&#x9996;&#x9875;&#x8DF3;&#x8F6C;&#x4E09;&#x6B21;&#x4EE5;&#x5185;&#x80FD;&#x591F;&#x8DF3;&#x5230;&#x7684;&#x9875;&#x9762;&#x624D;&#x80FD;&#x88AB;&#x6293;&#x53D6;&#x5230;&#x3002;</p>
<p><strong>&#x7EC3;&#x4E60; 8.7&#xFF1A;</strong> &#x5B8C;&#x6210;&#x4E00;&#x4E2A;&#x5E76;&#x53D1;&#x7A0B;&#x5E8F;&#x6765;&#x521B;&#x5EFA;&#x4E00;&#x4E2A;&#x7EBF;&#x4E0A;&#x7F51;&#x7AD9;&#x7684;&#x672C;&#x5730;&#x955C;&#x50CF;&#xFF0C;&#x628A;&#x8BE5;&#x7AD9;&#x70B9;&#x7684;&#x6240;&#x6709;&#x53EF;&#x8FBE;&#x7684;&#x9875;&#x9762;&#x90FD;&#x6293;&#x53D6;&#x5230;&#x672C;&#x5730;&#x786C;&#x76D8;&#x3002;&#x4E3A;&#x4E86;&#x7701;&#x4E8B;&#xFF0C;&#x6211;&#x4EEC;&#x8FD9;&#x91CC;&#x53EF;&#x4EE5;&#x53EA;&#x53D6;&#x51FA;&#x73B0;&#x5728;&#x8BE5;&#x57DF;&#x4E0B;&#x7684;&#x6240;&#x6709;&#x9875;&#x9762;&#xFF08;&#x6BD4;&#x5982;golang.org&#x5F00;&#x5934;&#xFF0C;&#x8BD1;&#x6CE8;&#xFF1A;&#x5916;&#x94FE;&#x7684;&#x5E94;&#x8BE5;&#x5C31;&#x4E0D;&#x7B97;&#x4E86;&#x3002;&#xFF09;&#x5F53;&#x7136;&#x4E86;&#xFF0C;&#x51FA;&#x73B0;&#x5728;&#x9875;&#x9762;&#x91CC;&#x7684;&#x94FE;&#x63A5;&#x4F60;&#x4E5F;&#x9700;&#x8981;&#x8FDB;&#x884C;&#x4E00;&#x4E9B;&#x5904;&#x7406;&#xFF0C;&#x4F7F;&#x5176;&#x80FD;&#x591F;&#x5728;&#x4F60;&#x7684;&#x955C;&#x50CF;&#x7AD9;&#x70B9;&#x4E0A;&#x8FDB;&#x884C;&#x8DF3;&#x8F6C;&#xFF0C;&#x800C;&#x4E0D;&#x662F;&#x6307;&#x5411;&#x539F;&#x59CB;&#x7684;&#x94FE;&#x63A5;&#x3002;</p>
<p><strong>&#x8BD1;&#x6CE8;&#xFF1A;</strong>
&#x62D3;&#x5C55;&#x9605;&#x8BFB; <a href="http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/" target="_blank">Handling 1 Million Requests per Minute with Go</a>&#x3002;</p>

                                
                                </section>
                            
                        </div>
                    </div>
                
            </div>

            
                
                <a href="ch8-05.html" class="navigation navigation-prev " aria-label="Previous page: 并发的循环">
                    <i class="fa fa-angle-left"></i>
                </a>
                
                
                <a href="ch8-07.html" class="navigation navigation-next " aria-label="Next page: 基于select的多路复用">
                    <i class="fa fa-angle-right"></i>
                </a>
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"title":"示例: 并发的Web爬虫","level":"1.9.6","depth":2,"next":{"title":"基于select的多路复用","level":"1.9.7","depth":2,"path":"ch8/ch8-07.md","ref":"ch8/ch8-07.md","articles":[]},"previous":{"title":"并发的循环","level":"1.9.5","depth":2,"path":"ch8/ch8-05.md","ref":"ch8/ch8-05.md","articles":[]},"dir":"ltr"},"config":{"plugins":["katex","-search","-lunr"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"katex":{},"highlight":{},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"preface.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Go语言圣经","language":"zh-hans","gitbook":"3.x","description":"<The Go Programming Language>中文版"},"file":{"path":"ch8/ch8-06.md","mtime":"2020-08-13T07:15:34.337Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2020-08-13T07:17:23.922Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    

    </body>
</html>

